# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
from datetime import datetime
from scrapy.exporters import JsonItemExporter, CsvItemExporter
import redis
import pymongo

# 数据源管道
class AqiDataPipeline(object):
    def process_item(self, item, spider):
        # 抓取时间  当前时区时间
        item['crawl_item'] = datetime.now()
        # 0时区
        # item['crawl_item'] = datetime.utcnow()
        return item

# json管道 和csv的差不多 只是self.write = **JsonItemExporter**CsvItemExporter**(self.file) 不一样
class AqiJsonPipeline():
    def open_spider(self, spider):
        self.file = open('aqi.json', 'wb')
        self.write = JsonItemExporter(self.file)
        self.write.start_exporting()

    def process_item(self, item, spider):
        self.write.export_item(item)
        return item

    def close_spider(self, spider):
        self.write.finish_exporting()
        self.file.close()


# csv管道
class AqiCsvPipeline():
    def open_spider(self, spider):
        self.file = open('aqi.csv', 'wb')
        self.write = CsvItemExporter(self.file)
        self.write.start_exporting()

    def process_item(self, item, spider):
        self.write.export_item(item)
        return item

    def close_spider(self, spider):
        self.write.finish_exporting()
        self.file.close()

# redis 管道　存储key
class AqiRedisPipeline():
    def open_spider(self, spider):
        # 链接数据库
        self.cline = redis.StrictRedis(host='127.0.0.1', port=6379)
        # 存储的key
        self.save_key = 'aqi_redis'

    def process_item(self, item, spider):
        # 从左边添加一个值
        self.cline.lpush(self.save_key, dict(item))
        return item

class AqiMongodbPipeline():
    def open_spider(self, spider):
        # 链接数据库
        self.cline = pymongo.MongoClient(host='127.0.0.1', port=27017)
        # 数据库名字 和 集合的名字
        self.collection = self.cline.AQI.aqi

    def process_item(self, item, spider):
        self.collection.insert(dict(item))
        return item



















